'''
* This is the projet for Brtc LlmOps Platform
* @Author Leon-liao <liaosiliang@alltman.com>
* @Description //TODO 
* @File: 13_study_use_zh_splitter.py
* @Time: 2025/10/29
* @All Rights Reserve By Brtc
'''
from langchain_community.document_loaders import UnstructuredMarkdownLoader
from langchain_text_splitters import TextSplitter, CharacterTextSplitter, RecursiveCharacterTextSplitter

#`1、构建 markdown文档加载器 并获取 文档
loader = UnstructuredMarkdownLoader("./01.项目API文档.md")
documents = loader.load()

#2、构建分割器
text_splitter = RecursiveCharacterTextSplitter(
    separators=[
        "\n\n",
        "\n",
        "。|！|？",
        "\.\s|\!\s|\?\s",  # 英文标点符号后面通常需要加空格
        "；|;\s",
        "，|,\s",
        " ",
        ""
    ],
    chunk_size=500,
    chunk_overlap=50,
    add_start_index=True
)
#3、分割文档列表
chunks = text_splitter.split_documents(documents)
print(len(chunks))
max_len = 0
min_len = 1000000
for chunk in chunks:
    if max_len <  len(chunk.page_content):
        max_len = len(chunk.page_content)
    if min_len > len(chunk.page_content):
        min_len = len(chunk.page_content)
    print(f"块大小：{len(chunk.page_content)}, 元数据：{chunk.metadata}")
print(max_len)
print(min_len)